package readySiteScanners;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.*;

import AgentsDB.Agent;
import AgentsDB.AgentDataIO;
import AgentsDB.AgentUtils;

import searchEngine.SearchEngine;

public class IRR {
	static String objectFilePath = "d:\\ivan\\workspace\\SearchEngine\\WebContent\\agentSetObject";
	static URL url = null;
	{
		try {
			url = new URL("http://irr.by/realestate/rent-flats/longtime");
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace(System.out);
		} 
	}
	public static void main(String[] args)throws Exception
	{
		String filePath = "d:\\ivan\\workspace\\SearchEngine\\WebContent\\agents.data";
		String rentFlats = "http://irr.by/realestate/longtime/page_len100/page";
		String buyFlats = "http://irr.by/realestate/sale-flats/page_len100/page";
		String rentFlatsOffers = "http://irr.by/realestate/longtime/search/offertype=%D0%BF%D1%80%D0%B5%D0%B4%D0%BB%D0%BE%D0%B6%D0%B5%D0%BD%D0%B8%D0%B5/price=%D0%BC%D0%B5%D0%BD%D1%8C%D1%88%D0%B5+200/currency=USD/rooms=1/page_len100/page";
		String buyFlatsOffers = "http://irr.by/realestate/sale-flats/incity/search/offertype=%D0%BF%D1%80%D0%B5%D0%B4%D0%BB%D0%BE%D0%B6%D0%B5%D0%BD%D0%B8%D0%B5/price=%D0%BC%D0%B5%D0%BD%D1%8C%D1%88%D0%B5+43000/currency=USD/rooms=1/page_len100/page";
//      Searching agents and writing them tp database
//		searchAgents(rentFlats);
//		searchAgents(buyFlats);

		//Show database 
		//AgentsDB.AgentDataIO.readStoredAgents(objectFilePath);
		//AgentDataIO.writeAgentsToFile(filePath,objectFilePath);
		
		//search not agent messages urls
		Set<URL> urls =  IRR.searchMessages(buyFlatsOffers);
		  for (URL i: urls){
			  System.out.println ("<a href=\"" + i.toString()+"\">...</a>");
			  }
		
		
/*		
 * 		Manual agents input input
 * 		Set<Agent> agents = new HashSet<Agent>();
		agents.add(new Agent("Tvoya stolitca", Long.parseLong("172842724"), null, new Date()));
		agents.add(new Agent("Tvoya stolitca", Long.parseLong("172863737"), null, new Date()));
		AgentDataIO.ObjectStoreUpdate(agents, objectFilePath);*/
	}
	public static void searchAgents(String initialUrl){
		for (int j = 1; j<11; j++){
			String urlString = initialUrl+j;
			URL url = null;
			try{
				url = new URL(urlString);
			}catch(Exception e){
				e.printStackTrace(System.out);
			}
			StringBuilder stringUrlsFromMainPage = new StringBuilder();
			String separator = System.getProperty("line.separator");
			stringUrlsFromMainPage = SearchEngine.search("(realestate).{0,100}/\\d{6,10}/",url, separator);
			List<URL> urlList= new LinkedList<URL>();
			String[] urlsFromMainPage = stringUrlsFromMainPage.toString().split(separator);
			for(String iterator: urlsFromMainPage){
				try {
					urlList.add(new URL("http://irr.by/"+iterator.toString()));
				} catch (MalformedURLException e) {
					// TODO Auto-generated catch block
					e.printStackTrace(System.out);
				}
			}
			String agentMessageExpr = "<div class=\"b-owner\".*?(((>|-) {0,2}(Ð°|Ð)Ð³ÐµÐ½Ñ)|ÐÐÐ|Ð£ÐÐ|Ð Ð£Ð|ÐÐÐ|Ð§Ð Ð£Ð|Ð Ð§Ð£Ð).*?<div class=\"b-infAdvert\">";
			String agentNameExpr = "((ÐÐÐ|Ð£ÐÐ|Ð Ð£Ð|ÐÐÐ|Ð§Ð Ð£Ð|Ð Ð§Ð£Ð).*?<)|((Ð°|Ð)Ð³ÐµÐ½Ñ)";
			String agentPhoneExpr = "[2-6,9]{2}[), ,-]{0,2}\\d\\d\\d[ ,-]?\\d\\d[ ,-]?\\d\\d";
			HashSet<Agent> agents = (HashSet<Agent>) AgentUtils.searchAgents(urlList, agentMessageExpr, agentNameExpr, agentPhoneExpr);
			System.out.println("Writing object data file.");
			try {
				AgentDataIO.ObjectStoreUpdate(agents, objectFilePath);
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace(System.out);
			}
		}
		System.out.println("done");
	}
	public static Set<URL> searchMessages(String initialUrl){
		String separator = System.getProperty("line.separator");
		HashSet<URL> urlList= new HashSet<URL>();
		for (int j = 1; j<11; ++j){
			String urlString = initialUrl+j;
			URL url = null;
			try {
				url = new URL(urlString);
			} catch (MalformedURLException e) {
					System.out.println("Bad url: "+urlString);
					e.printStackTrace(System.out);
			}
			StringBuilder stringUrlsFromMainPage = new StringBuilder();
			stringUrlsFromMainPage = SearchEngine.search("(realestate).{0,100}/\\d{6,10}/",url, separator);
			String[] urlsFromMainPage = stringUrlsFromMainPage.toString().split(separator);
			for(String iterator: urlsFromMainPage){
				try {
					urlList.add(new URL("http://irr.by/"+iterator.toString()));
				} catch (MalformedURLException e) {
					System.out.println("Bad url: "+"http://irr.by/"+iterator.toString());
					e.printStackTrace(System.out);
				}
			}
		}	
		System.out.println("Found adv urls number: " + urlList.size());
		System.out.println("Sarching for messages");
		System.out.println("[--------------------]100%");
		System.out.print("[");

		int group = urlList.size() / 20;
		int i = 0;
		Iterator<URL> iterator = urlList.iterator();
		while (iterator.hasNext()) {
			boolean isAgent = false;
			StringBuilder temp = SearchEngine.search(
					"<div class=\"b-owner\".*?<div class=\"b-infAdvert\">",
					iterator.next(), separator);
			if (temp.length() > 1) {

				Matcher agentPhonePattern = Pattern
						.compile(
								"[2-6,9]{2}[), ,-]{0,2}\\d\\d\\d[ ,-]?\\d\\d[ ,-]?\\d\\d")
						.matcher(temp.toString());
				while (agentPhonePattern.find()) {
					// agentPhonePattern.group();
					// System.out.println(agentPhonePattern.group().replaceAll("[) ,-]",
					// ""));
					isAgent = AgentUtils.isAgent(agentPhonePattern.group()
							.replaceAll("[) ,-]", ""), objectFilePath);
				}

			}
			if (isAgent)
				iterator.remove();
			i += 1;
			if ((i % group) == 0)
				System.out.print("|");
		}
		System.out.println("]100%");
		System.out.print("done");

		return urlList;
	}
}
